# Program Purpose: The purpose of this program is to add two new computation features for the training file

from __future__ import division
from operator import itemgetter, attrgetter
import os
import sys
import math

print "Program Begins..."
# inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/MLRelated/Model15/TrainingSetForFixingTheMissingTOPKResults20130715_tfqAdded_tfcAdded_postingRankInDocAdd_postingRankInListAdd_WITH_headline.train"
# inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/Training_Set_2013_07_19_tfqAdded_tfcAdded_postingRankInDocAdd_postingRankInListAdd_WITH_headline.train"
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/MLRelated/Model17/TrainingSetForTOPKMissingResults20130719_tfqAdded_tfcAdded_postingRankInDocAdd_postingRankInListAdd_WITH_headline.train"
inputFileHandler = open(inputFileName,"r")
# outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/Training_Set_2013_07_15_tfqAdded_tfcAdded_postingRankInDocAdd_postingRankInListAdd_percentageForPostingRankInDoc_percentageForPostingRankInList_WITH_headline.train"
# outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/Training_Set_2013_07_19_tfqAdded_tfcAdded_postingRankInDocAdd_postingRankInListAdd_percentageForPostingRankInDoc_percentageForPostingRankInList_WITH_headline.train"
outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/MLRelated/Model17/TrainingSetForTOPKMissingResults20130719_tfqAdded_tfcAdded_postingRankInDocAdd_postingRankInListAdd_percentageForPostingRankInDoc_percentageForPostingRankInList_WITH_headline.train"
outputFileHandler = open(outputFileName,"w")

oldHeadLine = inputFileHandler.readline()
newHeadLine = oldHeadLine.strip() + " " + "percentageForPostingRankInDoc" + " " + "percentageForPostingRankInList" + "\n"

outputFileHandler.write(newHeadLine)
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    docWordsInFloatFormat = float(lineElements[10])
    lengthOfInvertedIndexInFloatFormat = float(lineElements[8])
    postingRankInDocInFloatFormat = float(lineElements[15])
    postingRankInListInFloatFormat = float(lineElements[16])
    percentageForPostingRankInDocInFloatFormat = postingRankInDocInFloatFormat / docWordsInFloatFormat
    percentageForPostingRankInListInFloatFormat = postingRankInListInFloatFormat / lengthOfInvertedIndexInFloatFormat
    if percentageForPostingRankInDocInFloatFormat > 1.0 or percentageForPostingRankInListInFloatFormat > 1.0:
        print "system error: percentage value > 1.0"
        print "percentageForPostingRankInDocInFloatFormat:",percentageForPostingRankInDocInFloatFormat
        print "percentageForPostingRankInListInFloatFormat:",percentageForPostingRankInListInFloatFormat
        exit(1)
    outputFileHandler.write(line.strip() + " " + str(percentageForPostingRankInDocInFloatFormat) + " " + str(percentageForPostingRankInListInFloatFormat) + "\n")
    
inputFileHandler.close()
outputFileHandler.close()
print "Overall Processing Statistics:"
print "inputFileName:",inputFileName
print "outputFileName:",outputFileName
print "Program Ends."